# -*- coding: utf-8 -*-
import json
import sys
import time
import pymysql
from handle_db import db_data
from function import *
from urllib import parse
import warnings
warnings.filterwarnings("ignore")

# 这个地方必须这么写 函数名：response
def response(flow):
    # 通过抓包软包软件获取请求的用户列表接口(用户列表接口)
    if 'aweme/v1/discover/search/?cursor=' in flow.request.url:

        # 数据的解析
        dict_json = json.loads(flow.response.text)
        # 获取关键词
        keyword = get_keyword(flow.request.url)
        user_list = dict_json['user_list']
        for userinfo in user_list:
            # 组建视频作者信息数据
            user_info = userinfo.get('user_info', '')
            user_info_data = {}
            user_info_data['uid'] = user_info.get('uid', '')
            user_info_data['keyword'] = keyword
            user_info_data['unique_id'] = user_info.get('unique_id', '')
            user_info_data['language'] = user_info.get('language', '')
            user_info_data['following_count'] = user_info.get('following_count', '')
            user_info_data['follower_count'] = user_info.get('follower_count', '')
            user_info_data['total_favorited'] = user_info.get('total_favorited', '')
            user_info_data['signature'] = user_info.get('signature', '')
            user_info_data['short_id'] = user_info.get('short_id', '')
            user_info_data['avatar_uri'] = user_info.get('avatar_uri', '')
            user_info_data['nickname'] = user_info.get('nickname', '')
            user_info_data['status'] = user_info.get('status', '')
            user_info_data['avatar_larger'] = ','.join(user_info['avatar_larger'].get('url_list', ''))
            user_info_data['region'] = user_info.get('region', '')
            user_info_data['birthday'] = user_info.get('birthday', '')
            user_info_data['share_qrcode_uri'] = user_info.get('share_qrcode_uri', '')
            user_info_data['unique_id_modify_time'] = user_info.get('unique_id_modify_time', '')
            user_info_data['user_rate'] = user_info.get('user_rate', '')
            user_info_data['gender'] = user_info.get('gender', '')
            user_info_data['bind_phone'] = user_info.get('bind_phone', '')
            user_info_data['live_agreement_time'] = user_info.get('live_agreement_time', '')
            user_info_data['live_agreement'] = user_info.get('live_agreement', '')
            user_info_data['create_time'] = user_info.get('create_time', '')
            user_info_data['aweme_count'] = user_info.get('aweme_count', '')
            user_info_data['favoriting_count'] = user_info.get('favoriting_count', '')
            # 视频作者信息插入视频作者信息表
            db_data.handle_data_add('douyin_video_author', user_info_data)

    # 通过抓包软包软件获取请求的话题列表接口(话题列表接口)
    if 'aweme/v1/challenge/search/?cursor=0&keyword=' in flow.request.url:

        # 数据的解析
        dict_json = json.loads(flow.response.text)
        # 获取关键词
        keyword = get_keyword(flow.request.url)
        challenge_list = dict_json['challenge_list']
        for challengeinfo in challenge_list:
            # 组建话题视频作者信息数据
            user_info = challengeinfo.get('challenge_info', '')
            user_info_data = {}
            # 当前话题信息
            user_info_data['keyword'] = keyword
            user_info_data['cha_name'] = user_info.get('cha_name', '')
            user_info_data['view_count'] = user_info.get('view_count', '')
            user_info_data['user_count'] = user_info.get('user_count', '')
            user_info_data['cid'] = user_info.get('cid', '')
            # 作者信息
            author_info = user_info['author']
            user_info_data['uid'] = author_info.get('uid', '')
            user_info_data['unique_id'] = author_info.get('unique_id', '')
            user_info_data['language'] = author_info.get('language', '')
            user_info_data['signature'] = author_info.get('signature', '')
            user_info_data['short_id'] = author_info.get('short_id', '')
            user_info_data['avatar_uri'] = author_info.get('avatar_uri', '')
            user_info_data['nickname'] = author_info.get('nickname', '')
            user_info_data['status'] = author_info.get('status', '')
            user_info_data['avatar_larger'] = ','.join(author_info['avatar_larger'].get('url_list', ''))
            user_info_data['region'] = author_info.get('region', '')
            user_info_data['birthday'] = author_info.get('birthday', '')
            user_info_data['share_qrcode_uri'] = author_info.get('share_qrcode_uri', '')
            user_info_data['unique_id_modify_time'] = author_info.get('unique_id_modify_time', '')
            user_info_data['user_rate'] = author_info.get('user_rate', '')
            user_info_data['gender'] = author_info.get('gender', '')
            user_info_data['bind_phone'] = author_info.get('bind_phone', '')
            user_info_data['live_agreement_time'] = author_info.get('live_agreement_time', '')
            user_info_data['live_agreement'] = author_info.get('live_agreement', '')
            user_info_data['create_time'] = user_info.get('create_time', '')
            # 话题视频作者信息插入话题作者信息表
            db_data.handle_data_add('douyin_video_topic', user_info_data)

    # 通过抓包软包软件获取请求的接口(视频评论接口)
    if 'aweme/v2/comment/list/?aweme_id=' in flow.request.url:

        # 数据的解析
        dict_json = json.loads(flow.response.text)
        # 组建视频信息数据
        video_data = {}
        video_data['has_more'] = dict_json.get('has_more', 0)
        video_data['reply_style'] = dict_json.get('reply_style', 0)
        video_data['total'] = dict_json.get('total', 0)
        video_data['now'] = dict_json['extra']['now']
        try:
            video_aweme_id = dict_json['comments'][0]['aweme_id']
        except:
            video_aweme_id=''

        print('视频评论接口:视频信息数据更新到视频信息数据表')
        # 视频信息数据更新到视频信息数据表
        db_data.handle_data_update('douyin_video', 'aweme_id', video_aweme_id, '\'', video_data)

        # 组建评论信息数据
        for comm in dict_json['comments']:
            comm_data = {}
            comm_data['create_time'] = comm.get('create_time', '')
            comm_data['now'] = video_data.get('now', '')
            comm_data['aweme_id'] = comm.get('aweme_id', '')
            comm_data['cid'] = comm.get('cid', '')
            comm_data['status'] = comm.get('status', '')
            comm_data['text'] = comm.get('text', '')
            comm_data['digg_count'] = comm.get('digg_count', '')
            comm_data['reply_comment_total'] = comm.get('reply_comment_total', '0')
            # 评论信息插入视频评论表
            print('走到了视频评论列表111111111')
            db_data.handle_data_add('douyin_comment', comm_data)
            # 组建评论用户信息
            user = comm['user']
            user_data = {}
            user_data['cid'] = comm.get('cid', '')
            user_data['language'] = user.get('language', '')
            user_data['signature'] = user.get('signature', '')
            user_data['short_id'] = user.get('short_id', '')
            user_data['unique_id'] = user.get('unique_id', '')
            user_data['avatar_uri'] = user.get('avatar_uri', '')
            user_data['nickname'] = user.get('nickname', '')
            user_data['status'] = user.get('status', '')
            user_data['avatar_larger'] = user['avatar_larger'].get('url_list', '')[0]
            user_data['uid'] = user.get('uid', '')
            user_data['region'] = user.get('region', '')
            user_data['birthday'] = user.get('birthday', '')
            user_data['share_qrcode_uri'] = user.get('share_qrcode_uri', '')
            user_data['unique_id_modify_time'] = user.get('unique_id_modify_time', '')
            user_data['user_rate'] = user.get('user_rate', '')
            user_data['gender'] = user.get('gender', '')
            user_data['bind_phone'] = user.get('bind_phone', '')
            user_data['live_agreement_time'] = user.get('live_agreement_time', '')
            user_data['live_agreement'] = user.get('live_agreement', '')
            user_data['type'] = 'comment'
            # 评论用户信息插入评论、回复用户信息表
            db_data.handle_data_add('douyin_comment_user', user_data)
            # 组建评论回复信息
            if comm['reply_comment'] is None:
                pass
            else:
                for reply in comm['reply_comment']:
                    reply_data = {}
                    reply_data['reply_id'] = reply.get('reply_id', '')
                    reply_data['cid'] = reply.get('cid', '')
                    reply_data['aweme_id'] = reply.get('aweme_id', '')
                    reply_data['text'] = reply.get('text', '')
                    reply_data['status'] = reply.get('status', '')
                    reply_data['digg_count'] = reply.get('digg_count', '')
                    reply_data['create_time'] = reply.get('create_time', '')
                    # 评论回复信息插入评论回复信息表
                    db_data.handle_data_add('douyin_comment_reply', reply_data)
                    #  组建评论回复的用户信息
                    reply_user = reply['user']
                    reply_user_data = {}
                    reply_user_data['cid'] = reply.get('cid', '')
                    reply_user_data['language'] = reply_user.get('language', '')
                    reply_user_data['signature'] = reply_user.get('signature', '')
                    reply_user_data['short_id'] = reply_user.get('short_id', '')
                    reply_user_data['unique_id'] = reply_user.get('unique_id', '')
                    reply_user_data['avatar_uri'] = reply_user.get('avatar_uri', '')
                    reply_user_data['nickname'] = reply_user.get('nickname', '')
                    reply_user_data['status'] = reply_user.get('status', '')
                    reply_user_data['avatar_larger'] = reply_user['avatar_larger'].get('url_list', '')[0]
                    reply_user_data['uid'] = reply_user.get('uid', '')
                    reply_user_data['region'] = reply_user.get('region', '')
                    reply_user_data['birthday'] = reply_user.get('birthday', '')
                    reply_user_data['share_qrcode_uri'] = reply_user.get('share_qrcode_uri', '')
                    reply_user_data['unique_id_modify_time'] = reply_user.get('unique_id_modify_time', '')
                    reply_user_data['user_rate'] = reply_user.get('user_rate', '')
                    reply_user_data['gender'] = reply_user.get('gender', '')
                    reply_user_data['bind_phone'] = reply_user.get('bind_phone', '')
                    reply_user_data['live_agreement_time'] = reply_user.get('live_agreement_time', '')
                    reply_user_data['live_agreement'] = reply_user.get('live_agreement', '')
                    reply_user_data['type'] = 'reply'
                    # 评论回复的用户信息插入评论、回复用户信息表
                    db_data.handle_data_add('douyin_comment_user', reply_user_data)

    # 通过抓包软包软件获取请求的接口(评论以及回复用户个人中心接口)
    # if 'aweme/v1/user/?user_id=' in flow.request.url:
    #     # 数据的解析
    #     dict_json = json.loads(flow.response.text)
    #     # 组建个人信息数据
    #     user = dict_json.get('user', '')
    #     user_info = {}
    #     user_info['following_count'] = user.get('following_count', '')
    #     user_info['follower_count'] = user.get('follower_count', '')
    #     user_info['total_favorited'] = user.get('total_favorited', '')
    #     user_info['location'] = user.get('location', '')
    #     user_info['aweme_count'] = user.get('aweme_count', '')
    #     user_info['dongtai_count'] = user.get('dongtai_count', '')
    #     user_info['favoriting_count'] = user.get('favoriting_count', '')
    #     user_uid = user.get('uid', '')
    #     print('评论以及回复用户个人中心接口')
    #     # 评论以及回复用户信息数据更新到评论、回复用户信息表
    #     db_data.handle_data_update('douyin_comment_user', 'uid', user_uid, '\'', user_info)

    # 通过抓包软包软件获取请求的接口(作者个人中心视频列表接口)
    if 'aweme/v1/aweme/post/' in flow.request.url:
        # 数据的解析
        dict_json = json.loads(flow.response.text)
        # 只能存储作者的，其他用户的不存储
        # 获取任意表里面的单个字段的集合(get_field:需要获取信息的字段, table:表明)
        # 组建视频信息数据
        for video in dict_json.get('aweme_list', ''):
            # 获取用户uid,也就是分享id,然后取出使用(field:需要取出的字段,table:表名)
            author_uid = db_data.get_table_info('uid', 'douyin_video_author')
            author_user_id = str(video.get('author_user_id', ''))
            print(author_uid[0],'00000000')
            # 二维元组变一维列表
            list_author_uid = handle_tuple(author_uid)
            # 如果uid存在于作者列表中中，再提取视频相应的数据
            if author_user_id in list_author_uid:
                # 组建视频信息数据
                video_data = {}
                video_data['aweme_id'] = video.get('aweme_id', '')
                video_data['author_user_id'] = video.get('author_user_id', '')
                video_data['desc'] = video.get('desc', '')
                video_data['create_time'] = video.get('create_time', '')
                video_data['digg_count'] = video['statistics'].get('digg_count', '')
                video_data['share_count'] = video['statistics'].get('share_count', '')
                video_data['forward_count'] = video['statistics'].get('forward_count', '')
                video_data['comment_count'] = video['statistics'].get('comment_count', '')
                video_data['share_url'] = video.get('share_url', '')
                video_data['rate'] = video.get('rate', '')
                video_data['region'] = video.get('region', '')
                video_data['duration'] = video.get('duration', '')
                if video['video'] is None:
                    pass
                else:
                    video_data['play_addr'] = ','.join(video['video']['play_addr_lowbr']['url_list'][0:2])
                    video_data['origin_cover'] = ','.join(video['video']['origin_cover']['url_list'])
                    video_data['ratio'] = video['video'].get('ratio', '')
                video_data['group_id'] = video.get('group_id', '')
                video_data['reviewed'] = video['status'].get('reviewed', '')
                video_data['allow_share'] = video['status'].get('allow_share', '')
                video_data['private_status'] = video['status'].get('private_status', '')
                video_data['download_status'] = video['status'].get('download_status', '')
                video_data['allow_comment'] = video['status'].get('allow_comment', '')
                # 视频信息数据插入到视频信息数据表
                db_data.handle_data_add('douyin_video', video_data)

                # 组建视频音乐信息数据
                music = video.get('music', '')
                music_data = {}
                # 如果是字典了再往下走
                if type(music).__name__ == 'dict':
                    music_data['mid'] = music.get('mid', '')
                    music_data['id'] = music.get('id', '')
                    music_data['aweme_id'] = video.get('aweme_id', '')
                    music_data['title'] = music.get('title', '')
                    music_data['status'] = music.get('status', '')
                    music_data['owner_handle'] = music.get('owner_handle', '')
                    music_data['owner_nickname'] = music.get('owner_nickname', '')
                    music_data['owner_id'] = music.get('owner_id', '')
                    music_data['duration'] = music.get('duration', '')
                    music_data['play_url'] = music.get('play_url', '').get('uri', '')
                    music_data['source_platform'] = music.get('source_platform', '')
                    # 视频音乐信息插入视频音乐信息表
                    db_data.handle_data_add('douyin_video_music', music_data)

    # 通过抓包软包软件获取请求的接口(作者话题视频列表接口)
    if 'aweme/v1/challenge/aweme/?ch_id=' in flow.request.url:
        # 数据的解析
        dict_json = json.loads(flow.response.text)
        # 只能存储作者的，其他用户的不存储
        # 获取任意表里面的单个字段的集合(get_field:需要获取信息的字段, table:表明)
        # 组建话题视频信息数据
        for video in dict_json.get('aweme_list', ''):
            # 获取用户uid,也就是分享id,然后取出使用(field:需要取出的字段,table:表名)
            author_cid = db_data.get_table_info('cid', 'douyin_video_topic')
            cid = str(video['cha_list'][0]['cid'])
            hashtag_name = str(video['cha_list'][0]['cha_name'])

            # print(author_cha_name[0],hashtag_name, '00000000')
            # 二维元组变一维列表
            list_author_cid = handle_tuple(author_cid)
            # 如果uid存在于作者列表中中，再提取视频相应的数据
            if cid in list_author_cid:
                print(hashtag_name, '1111')
                # 组建视频信息数据
                video_data = {}
                video_data['aweme_id'] = video.get('aweme_id', '')
                video_data['hashtag_name'] = hashtag_name
                video_data['top_cid'] = cid
                video_data['author_user_id'] = video.get('author_user_id', '')
                video_data['desc'] = video.get('desc', '')
                video_data['create_time'] = video.get('create_time', '')
                video_data['digg_count'] = video['statistics'].get('digg_count', '')
                video_data['share_count'] = video['statistics'].get('share_count', '')
                video_data['forward_count'] = video['statistics'].get('forward_count', '')
                video_data['comment_count'] = video['statistics'].get('comment_count', '')
                video_data['share_url'] = video.get('share_url', '')
                video_data['rate'] = video.get('rate', '')
                video_data['region'] = video.get('region', '')
                if video['video'] is None:
                    pass
                else:
                    video_data['play_addr'] = ','.join(video['video']['play_addr_lowbr']['url_list'][0:2])
                    video_data['origin_cover'] = ','.join(video['video']['origin_cover']['url_list'])
                    video_data['ratio'] = video['video'].get('ratio', '')
                video_data['group_id'] = video.get('group_id', '')
                video_data['reviewed'] = video['status'].get('reviewed', '')
                video_data['allow_share'] = video['status'].get('allow_share', '')
                video_data['private_status'] = video['status'].get('private_status', '')
                video_data['download_status'] = video['status'].get('download_status', '')
                video_data['allow_comment'] = video['status'].get('allow_comment', '')
                # 话题视频信息数据插入到视频信息数据表
                db_data.handle_data_add('douyin_video', video_data)

                # 组建话题视频音乐信息数据
                music = video.get('music', '')
                music_data = {}
                # 如果是字典了再往下走
                if type(music).__name__ == 'dict':
                    music_data['mid'] = music.get('mid', '')
                    music_data['id'] = music.get('id', '')
                    music_data['aweme_id'] = video.get('aweme_id', '')
                    music_data['title'] = music.get('title', '')
                    music_data['status'] = music.get('status', '')
                    music_data['owner_handle'] = music.get('owner_handle', '')
                    music_data['owner_nickname'] = music.get('owner_nickname', '')
                    music_data['owner_id'] = music.get('owner_id', '')
                    music_data['duration'] = music.get('duration', '')
                    music_data['play_url'] = music.get('play_url', '').get('uri', '')
                    music_data['source_platform'] = music.get('source_platform', '')
                    music_data['hashtag_name'] = hashtag_name
                    # 话题视频音乐信息插入视频音乐信息表
                    db_data.handle_data_add('douyin_video_music', music_data)

    # 通过抓包软包软件获取请求的粉丝用户数据接口(粉丝用户数据接口)
    if 'aweme/v1/user/follower/list/?user_id=' in flow.request.url:

        # 数据的解析
        dict_json = json.loads(flow.response.text)
        # 获取关键词
        author_uid = get_author_uid(flow.request.url)
        myself_user_id = dict_json['myself_user_id']

        followers_list = dict_json['followers']
        for user_info in followers_list:
            # 组建粉丝用户数据
            user_info_data = {}
            user_info_data['uid'] = user_info.get('uid', '')
            user_info_data['author_uid'] = author_uid
            user_info_data['myself_user_id'] = myself_user_id
            user_info_data['type'] = 'fans'
            user_info_data['unique_id'] = user_info.get('unique_id', '')
            user_info_data['language'] = user_info.get('language', '')
            user_info_data['signature'] = user_info.get('signature', '')
            user_info_data['short_id'] = user_info.get('short_id', '')
            user_info_data['avatar_uri'] = user_info.get('avatar_uri', '')
            user_info_data['nickname'] = user_info.get('nickname', '')
            user_info_data['status'] = user_info.get('status', '')
            user_info_data['avatar_larger'] = ','.join(user_info['avatar_larger'].get('url_list', ''))
            user_info_data['region'] = user_info.get('region', '')
            user_info_data['birthday'] = user_info.get('birthday', '')
            user_info_data['share_qrcode_uri'] = user_info.get('share_qrcode_uri', '')
            user_info_data['unique_id_modify_time'] = user_info.get('unique_id_modify_time', '')
            user_info_data['user_rate'] = user_info.get('user_rate', '')
            user_info_data['gender'] = user_info.get('gender', '')
            user_info_data['bind_phone'] = user_info.get('bind_phone', '')
            user_info_data['live_agreement_time'] = user_info.get('live_agreement_time', '')
            user_info_data['live_agreement'] = user_info.get('live_agreement', '')
            user_info_data['create_time'] = user_info.get('create_time', '')
            # 粉丝用户数据插入评论、回复用户信息表
            db_data.handle_data_add('douyin_comment_user', user_info_data)